"use strict";
var nconf = require('nconf');
nconf.argv().env();

var fs = require('fs'),
    os = require('os'),
    winston = require('winston'),
	async = require('async'),
	mongodb = require('mongodb'),
    http = require('http'),
    cheerio = require("cheerio"),
    url = require("url");


var server = new mongodb.Server("127.0.0.1",27017,{safe:true});//本地27017端口
var webpage = {};
var queue = [];
var filePath = "/root/zspiderlog.txt"

new mongodb.Db('zspider',server,{safe:true}).open(function(error,client){//数据库：zspider
	if(error) throw error;
    webpage = new mongodb.Collection(client,'webpage');//表 webpage
    var url = nconf.get('_')[0];

	main(url);
    queueCrtl();
});
var i = 0;
var j = 0;



function main(urlG) {
    var req = http.get(urlG, function(res) {
            var urlObj = url.parse(urlG);
            console.log(urlG);
            //console.log('HEADERS: ' + JSON.stringify(res.headers));
            var contentType = res.headers["content-type"];
            if(typeof(contentType) == "string" && contentType.match("text\/html")) {
                res.setEncoding('utf8');
                var html = '';
                res.on('data', function (chunk) {
                    html += chunk;
                });
                res.on('end', function() {
                    res.headers["body"] = html;
                    res.headers["url"] = urlG;

                    webpage.count({"url":urlG},function(err,cursor){
                        if(cursor > 0) {
                            console.warn("exist: " + urlG);
                            return;
                        }

                        console.log(i++);
                        webpage.insert(res.headers,function(err,doc){
                            if (err) console.warn(err.message);

                            // fs.appendFile(filePath, urlG+"\n", function (err) {
                            //     if (err) throw err;
                            //     //console.log('The "data to append" was appended to file!');
                            // });
                        });


                        var $ = cheerio.load(html);
                        $("a").each(function(){
                            var urlT = "";
                            var href = $(this).attr("href");
                            if(typeof(href) != "string" || typeof(href) == "string" && href.match("javascript:")) {
                                return;
                            }
                            var hrefObj = url.parse(href);
                            if(hrefObj.protocol == "http:" || urlObj.protocol == "https:") {
                                urlT = hrefObj.protocol+'\/\/'+hrefObj.host + hrefObj.path;
                            }
                            if(hrefObj.protocol == null) {
                                if(href.match(/^\//)) {
                                    urlT = urlObj.protocol+'\/\/'+urlObj.host+hrefObj.path;
                                } else if(!href.match(/^#/)) {
                                    urlT = urlObj.protocol+'\/\/'+urlObj.host+urlObj.pathname+'\/'+hrefObj.path;
                                }
                            }
                            if(!urlT) {
                                return;
                            }

                            webpage.count({"url":urlT},function(err,cursor){
                                if(cursor > 0) {
                                    return;
                                }
                                queue.push(urlT);
                                queue[urlT] = true;
                                urlT = null;
                            });                            

                            href = null;
                            hrefObj = null;
                        });
                        $ = null;
                        html = null;
                        urlObj = null;
                    });
                    contentType = null;
                });
            }            
    }).on('error', function(e) {
        console.log("Got error: " + e.message);
    }).on('close', function(e) {
        console.log("链接关闭"+j++);
    });
}

function queueCrtl() {

    setInterval(function(){
        if(queue.length == 0) {
            return;
        }

        var urlG = queue.shift();
        console.log(urlG);
        delete queue[urlG];
        main(urlG);
    },500);
}

function delHtmlTag(str)
{
    str = str.replace(/<[^>]+>/g,"");
    str = str.replace(/\s/g," ");
    return str;//去掉所有的html标记
}
// webpage.insert({"html":html},function(err,doc){console.log(111);});
